import os, time, random
from multiprocessing import Pool
from threading import Thread
from bestbuy_spider import BestbuySpider
from walmart_spider import WalmartSpider
from staples_spider import StaplesSpider
from model import Spider
import pdb
import sys

def str_to_class(str):
    return getattr(sys.modules[__name__], str)

class Manager:
  # def __init__(self,):
  #   self.spider_list = []

  # def add_spider(self, spider):
  #   self.spider_list.append(spider)

  def run(self):
    while(True):
      print("current time : %s" % time.ctime())
      threads = []
      spiders = Spider.query.filter_by(status=True).all()
      for sp in spiders:
        t = Thread(target=self.run_spider, args=(sp,))
        threads.append(t)
      for t in threads:
        t.start()
      for t in threads:
        t.join()
      # with Pool(4) as pool:
      #   pool.map(self.run_spider, spiders)
      time.sleep(10)

  def run_spider(self, spider):
    if(not spider.status):
        return
    spider_map = {
      'bestbuy': 'BestbuySpider',
      'walmart': 'WalmartSpider',
      'staples': 'StaplesSpider',
    }
    key_list = list(spider_map.keys())
    match_key = [key for key in key_list if key in spider.url][0]
    klass = str_to_class(spider_map[match_key])
    klass(spider).setToDb()

if __name__=='__main__':
  manager = Manager()
  manager.run()